import configparser
import argparse
import os
import os.path
import shutil
import sys
import random
import json
import datetime
import pandas as pd
from multiprocessing.pool import Pool

try:
    from StringIO import StringIO  # python2
except ImportError:
    from io import StringIO  # python3

################################################################################
USERAGENT = 'tis/download.py_1.0--' + sys.version.replace('\n', '').replace('\r', '')


def geturl(url, token=None, out=None):
    headers = {'user-agent': USERAGENT}
    if not token is None:
        headers['Authorization'] = 'Bearer ' + token
    try:
        import ssl
        CTX = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
        if sys.version_info.major == 2:
            import urllib2
            try:
                fh = urllib2.urlopen(urllib2.Request(url, headers=headers), context=CTX)
                if out is None:
                    return fh.read()
                else:
                    shutil.copyfileobj(fh, out)
            except urllib2.HTTPError as e:
                print('HTTP GET error code: %d' % e.code(), file=sys.stderr)
                print('HTTP GET error message: %s' % e.message, file=sys.stderr)
            except urllib2.URLError as e:
                print('Failed to make request: %s' % e.reason, file=sys.stderr)
            return None

        else:
            from urllib.request import urlopen, Request, URLError, HTTPError
            try:
                fh = urlopen(Request(url, headers=headers), context=CTX)
                if out is None:
                    return fh.read().decode('utf-8')
                else:
                    shutil.copyfileobj(fh, out)
            except HTTPError as e:
                print('HTTP GET error code: %d' % e.code(), file=sys.stderr)
                print('HTTP GET error message: %s' % e.message, file=sys.stderr)
            except URLError as e:
                print('Failed to make request: %s' % e.reason, file=sys.stderr)
            return None

    except AttributeError:
        # OS X Python 2 and 3 don't support tlsv1.1+ therefore... curl
        import subprocess
        try:
            args = ['curl', '--fail', '-sS', '-L', '--get', url]
            for (k, v) in headers.items():
                args.extend(['-H', ': '.join([k, v])])
            if out is None:
                # python3's subprocess.check_output returns stdout as a byte string
                result = subprocess.check_output(args)
                return result.decode('utf-8') if isinstance(result, bytes) else result
            else:
                subprocess.call(args, stdout=out)
        except subprocess.CalledProcessError as e:
            print('curl GET error message: %' + (e.message if hasattr(e, 'message') else e.output), file=sys.stderr)
        return None


################################################################################

DESC = "This script will recursively download all files if they don't exist from a LAADS URL and stores them to the specified path"


def sync(param):
    src = param[0]
    dest = param[1]
    tok = param[2]
    '''synchronize src url with dest directory'''
    try:
        import csv
        files = [f for f in csv.DictReader(StringIO(geturl('%s.csv' % src, tok)), skipinitialspace=True)]
    except ImportError:
        import json
        files = json.loads(geturl(src + '.json', tok))

    # use os.path since python 2/3 both support it while pathlib is 3.4+
    for f in files:
        # currently we use filesize of 0 to indicate directory
        filesize = int(f['size'])
        path = os.path.join(dest, f['name'])
        url = src + '/' + f['name']
        if filesize == 0:
            try:
                if not os.path.exists(path):
                    print('creating dir:', path)
                    os.mkdir(path)
                    sync(src + '/' + f['name'], path, tok)
                else:
                    sync(src + '/' + f['name'], path, tok)
            except IOError as e:
                print("mkdir `%s': %s" % (e.filename, e.strerror), file=sys.stderr)
                sys.exit(-1)
        else:
            try:
                if not os.path.exists(path):
                    print('downloading: ', path)
                    with open(path, 'w+b') as fh:
                        geturl(url, tok, fh)
                else:
                    if os.path.getsize(path) == 0:
                        os.remove(path)
                        print('downloading: ', path)
                        with open(path, 'w+b') as fh:
                            geturl(url, tok, fh)
                    else:
                        print('skipping: ', path)
            except IOError as e:
                print("open `%s': %s" % (e.filename, e.strerror), file=sys.stderr)
                sys.exit(-1)
    return True


def main_run(source, destination, token):
    if not os.path.exists(destination):
        os.makedirs(destination)
    return sync(source, destination, token)


if __name__ == '__main__':
    conf = configparser.ConfigParser()
    conf.read(os.getcwd() + '/config/config.ini', encoding="utf-8")
    modis_types = str(conf.get('MODIS', 'types')).split(',')
    url = conf.get('MODIS', 'url')
    tokens = ["E87BCD08-FE03-11EA-BFAA-A90C3E2C173E","ECAA0368-FE03-11EA-8B18-95B39A1A7C62",
             "F049DDB8-FE03-11EA-8C0B-AB5325B984A8","F5D20CE2-FE03-11EA-8062-B8710EF8D969",
             "F848F4CC-FE03-11EA-8C0B-AB5325B984A8"]
    destination = conf.get('MODIS', 'destination')
    start_year = conf.get('MODIS', 'start_year')
    end_year = conf.get('MODIS', 'end_year')
    current_time = datetime.datetime.now().strftime('%Y%m%d')
    for modis_type in modis_types:
        print("The currently downloaded MODIS data type is:{}".format(modis_type))
        if modis_type == 'MOD13Q1':
            time_frequency = 16
        else:
            time_frequency = 8
        param = []
        progress_dir = os.path.join(os.getcwd(), 'storage/progress')
        progress_name = progress_dir + '/progress_real_time_pool_{}.json'.format(modis_type)
        if not os.path.exists(progress_name):
            with open(progress_name, 'w') as of:
                json.dump(start_year, of)
        for year in range(int(start_year), int(end_year)):
            if year == 2000:
                start_day = 49
            else:
                start_day = 1
            for i in range(start_day, 367, time_frequency):
                day = str(i).rjust(3, '0')
                output_path = destination + "/{0}/{1}/{2}".format(modis_type, year, day)
                if not os.path.exists(output_path):
                    os.makedirs(output_path)
                source = url + "/{0}/{1}/{2}/".format(modis_type, year, day)
                param.append([source, output_path, random.choice(tokens)])
            with open(progress_name, 'r') as of:
                progress_time = json.load(of)
            if year >= int(progress_time):
                try:
                    with Pool(15) as pool:
                        pool.map(sync, param)
                    with open(progress_name, 'w') as of:
                        json.dump(str(year), of)
                except:
                    print("Unexpected error:", sys.exc_info())
                    continue



